import urllib
import codecs
import chardet
from lxml import etree
import lxml.html


# ==== ==== ==== ====
# GET feed
# ==== ==== ==== ====
# 1.
pages = ["http://delo.si/",
         "http://news.bbc.co.uk/"]

page = pages[1]

# 2. load and parse page
html = urllib.urlopen(page).read()
html = lxml.html.fromstring(html)

# 3. find rss link
#".//div[@class='article-text'][@name='article-text']"
#<link rel="alternate" type="application/rss+xml" title="Delo.si RSS" href="/rss" />
rssLink = html.find(".//link[@type='application/rss+xml']")
rssLink = rssLink.attrib['href']

# 3.1 prepare rss link
if not rssLink.startswith('http'):
    otherLink = "%s/%s" % (page.split("/")[2], rssLink)
    rssLink = "http://%s" % otherLink.replace("//", "/")

print rssLink

# ==== ==== ==== ====
# PARSE feed
# ==== ==== ==== ====
import feedparser

d = feedparser.parse(rssLink)
encoding = d.encoding
#updated = d.updated

data = {}
encodings = []

# get news
for i in d['entries']:
    print "*" * 20
    print i['title']
    print i['link']
    print i['updated_parsed']
    html = urllib.urlopen(i['link']).read()
    encoding = chardet.detect(html)
    encodings.append(encoding)
    data[i['link']] = html #.decode(encoding['encoding'])

foo = dict([(i, 0) for i in set([i['encoding'] for i in encodings])])

for i in encodings:
    foo[i['encoding']] += i['confidence']

for i in data:
    data[i] = data[i].decode
